﻿using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Net;
using System.IO;
using System.Net.Sockets;
using System.Text.RegularExpressions;
using System.Collections.ObjectModel;
using pageAnalyzer.Properties;
using System.Runtime.InteropServices;
using System.Reflection;
using System.IO.Compression;
using System.Data.SqlClient;
using System.Data;

namespace pageAnalyzer
{
    public static class Settings
    {
        private static string ConfigurationFilePath;
        private static string folder;

        static Settings()
        {
            folder = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
            ConfigurationFilePath = Path.Combine(folder, "config.ini");
        }

        /// <summary>
        /// 线程数
        /// </summary>
        public static int ThreadCount
        {
            get
            {
                return Convert.ToInt32(GetValue("ThreadCount", 50));
            }
            set
            {
                SetValue("ThreadCount", value);
            }
        }

        /// <summary>
        /// 若队列为空,线程睡觉的时间(秒)
        /// </summary>
        public static int ThreadSleepTimeWhenQueueIsEmpty
        {
            get
            {
                return Convert.ToInt32(GetValue("ThreadSleepTimeWhenQueueIsEmpty", 2));
            }
            set
            {
                SetValue("ThreadSleepTimeWhenQueueIsEmpty", value);
            }
        }

        /// <summary>
        /// 连接超时
        /// </summary>
        public static int ConnectionTimeout
        {
            get
            {
                return Convert.ToInt32(GetValue("ConnectionTimeout", 20));
            }
            set
            {
                SetValue("ConnectionTimeout", value);
            }
        }

        /// <summary>
        /// 默认存储方式 (0:文件系统；1:SQLite.)
        /// </summary>
        public static string DataStoreMode
        {
            get
            {
                return Convert.ToString(GetValue("DataStoreMode", "0"));
            }
            set
            {
                SetValue("DataStoreMode", value);
            }
        }

        /// <summary>
        /// SQLite文件名
        /// </summary>
        public static string SQLiteDBFolder
        {
            get
            {
                return Convert.ToString(GetValue("SQLiteDBFolder", "crawlerdb.s3db"));
            }
            set
            {
                SetValue("SQLiteDBFolder", value);
            }
        }

        /// <summary>
        /// 下载文件夹
        /// </summary>
        public static string FileSystemFolder
        {
            get
            {
                return Convert.ToString(GetValue("FileSystemFolder", folder));
            }
            set
            {
                SetValue("FileSystemFolder", value);
            }
        }

        /// <summary>
        /// 
        /// </summary>
        public static bool AllowAllMimeTypes
        {
            get
            {
                return Convert.ToBoolean(GetValue("AllowAllMimeTypes", true));
            }
            set
            {
                SetValue("AllowAllMimeTypes", value);
            }
        }

        /// <summary>
        /// 会下载的MIME类型,默认只支持网页. 如选多种，用逗号 "," 分隔.
        /// </summary>
        public static string FileMatches
        {
            get
            {
                return Convert.ToString(GetValue("FileMatches", ""));
            }
            set
            {
                SetValue("FileMatches", value);
            }
        }

        /// <summary>
        /// 未使用
        /// </summary>
        public static string HighPriority
        {
            get
            {
                return Convert.ToString(GetValue("HighPriority", ""));
            }
            set
            {
                SetValue("HighPriority", value);
            }
        }

        /// <summary>
        /// 语言
        /// </summary>
        public static string Language
        {
            get
            {
                return Convert.ToString(GetValue("Language", ""));
            }
            set
            {
                SetValue("Language", value);
            }
        }

        static void SetValue(string keyName, object value)
        {
            NativeMethods.WritePrivateProfileString("Crawler", keyName, value.ToString(), ConfigurationFilePath);
        }

        static object GetValue(string keyName, object defaultValue)
        {
            StringBuilder retVal = new StringBuilder(1024);
            NativeMethods.GetPrivateProfileString("Crawler", keyName, defaultValue.ToString(), retVal, 1024, ConfigurationFilePath);
            return retVal.ToString();
        }
    }

    class NativeMethods
    {
        [DllImport("kernel32")]
        internal static extern long WritePrivateProfileString(
            string appName,
            string keyName,
            string value,
            string fileName);

        [DllImport("kernel32")]
        internal static extern int GetPrivateProfileString(
            string appName,
            string keyName,
            string _default,
            StringBuilder returnedValue,
            int size,
            string fileName);

    }
    public class NWebResponse
    {
        public Socket socket;
        public Uri ResponseUri;
        public WebHeaderCollection Headers;
        public string Header;
        public string ContentType;
        public int ContentLength;
        public bool KeepAlive;

        public NWebResponse()
        {
        }

        public void Connect(NWebRequest request)
        {
            ResponseUri = request.RequestUri;

            socket = new Socket(AddressFamily.InterNetwork, SocketType.Stream, ProtocolType.Tcp);
            IPEndPoint remoteEP = new IPEndPoint(Dns.GetHostEntry(ResponseUri.Host).AddressList[0], ResponseUri.Port);
            try
            {
                socket.Connect(remoteEP);
            }
            catch (Exception e)
            {

            }
        }

        public void SetTimeout(int Timeout)
        {
            socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.SendTimeout, Timeout * 1000);
            socket.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReceiveTimeout, Timeout * 1000);
        }

        public void SendRequest(NWebRequest request)
        {
            ResponseUri = request.RequestUri;

            request.Header = request.Method + " " + ResponseUri.PathAndQuery + " HTTP/1.0\r\n" + request.Headers;
            socket.Send(Encoding.ASCII.GetBytes(request.Header));
        }

        public void ReceiveHeader()
        {
            Header = "";
            Headers = new WebHeaderCollection();

            byte[] bytes = new byte[10];
            while (socket.Receive(bytes, 0, 1, SocketFlags.None) > 0)
            {
                Header += Encoding.ASCII.GetString(bytes, 0, 1);
                if (bytes[0] == '\n' && Header.EndsWith("\r\n\r\n"))
                    break;
            }
            MatchCollection matches = new Regex("[^\r\n]+").Matches(Header.TrimEnd('\r', '\n'));
            for (int n = 1; n < matches.Count; n++)
            {
                string[] strItem = matches[n].Value.Split(new char[] { ':' }, 2);
                if (strItem.Length > 0)
                    Headers[strItem[0].Trim()] = strItem[1].Trim();
            }
            // check if the page should be transfered to another location
            if (matches.Count > 0 && (
                matches[0].Value.IndexOf(" 302 ") != -1 ||
                matches[0].Value.IndexOf(" 301 ") != -1))
                // check if the new location is sent in the "location" header
                if (Headers["Location"] != null)
                {
                    try { ResponseUri = new Uri(Headers["Location"]); }
                    catch { ResponseUri = new Uri(ResponseUri, Headers["Location"]); }
                }
            ContentType = Headers["Content-Type"];
            if (Headers["Content-Length"] != null)
                ContentLength = int.Parse(Headers["Content-Length"]);
            KeepAlive = (Headers["Connection"] != null && Headers["Connection"].ToLower() == "keep-alive") ||
                        (Headers["Proxy-Connection"] != null && Headers["Proxy-Connection"].ToLower() == "keep-alive");
        }

        public void Close()
        {
            socket.Close();
        }

        public byte[] GetResponseStream()
        {
            MemoryStream streamOut = new MemoryStream();
            {
                using (BinaryWriter writer = new BinaryWriter(streamOut))
                {
                    // receive response buffer
                    byte[] RecvBuffer = new byte[10240];
                    int nBytes, nTotalBytes = 0;
                    // loop to receive response buffer
                    while ((nBytes = socket.Receive(RecvBuffer, 0, 10240, SocketFlags.None)) > 0)
                    {
                        // increment total received bytes
                        nTotalBytes += nBytes;
                        // write received buffer to file
                        writer.Write(RecvBuffer, 0, nBytes);
                        // check if connection Keep-Alive to can break the loop if response completed
                        if (KeepAlive && nTotalBytes >= ContentLength && ContentLength > 0)
                            break;
                    }
                }
            }
            return streamOut.ToArray();
        }
    }

    public class NWebRequest
    {
        public string Method;
        public string Header;
        public WebHeaderCollection Headers;
        public Uri RequestUri;
        public int Timeout;
        public bool KeepAlive;
        public NWebResponse response;

        public NWebRequest(Uri uri, bool bKeepAlive)
        {
            Headers = new WebHeaderCollection();
            RequestUri = uri;
            Headers["Host"] = uri.Host;
            KeepAlive = bKeepAlive;
            if (KeepAlive)
                Headers["Connection"] = "Keep-Alive";
            Method = "GET";
        }

        public NWebResponse GetResponse()
        {
            if (response == null || response.socket == null || response.socket.Connected == false)
            {
                response = new NWebResponse();
                response.Connect(this);
                response.SetTimeout(Timeout);
            }
            response.SendRequest(this);
            response.ReceiveHeader();
            return response;
        }

    }

    static class MemCache
    {
        public static Collection<string> AllowedFileTypes;
        public static int ConnectionTimeoutMs;
        public static string SQLiteDBFolder;
        public static string FileSystemFolder;
        public static bool AllowAllMimeTypes;
        public static int ThreadCount;
        public static int ThreadSleepTimeWhenQueueIsEmptyMs;

        static MemCache()
        {
            AllowedFileTypes = new Collection<string>(Settings.FileMatches.Split(','));
            ConnectionTimeoutMs = Settings.ConnectionTimeout * 1000;
            SQLiteDBFolder = Settings.SQLiteDBFolder;
            FileSystemFolder = Settings.FileSystemFolder;
            AllowAllMimeTypes = Settings.AllowAllMimeTypes;
            ThreadCount = Settings.ThreadCount;
            ThreadSleepTimeWhenQueueIsEmptyMs = Settings.ThreadSleepTimeWhenQueueIsEmpty * 1000;
        }
    }

    public class Parser
    {

        public static string[] ExtractLinks(string baseUri, string html, string domainUrl)
        {
            Collection<string> urls = new Collection<string>();

            try
            {
                string strRef = @"(href|HREF|src|SRC)[ ]*=[ ]*[""'][^""'#>]+[""']";
                MatchCollection matches = new Regex(strRef).Matches(html);

                foreach (Match match in matches)
                {
                    strRef = match.Value.Substring(match.Value.IndexOf('=') + 1).Trim('"', '\'', '#', ' ', '>');
                    try
                    {
                        if (IsGoodUri(strRef) && IsDomainSpecified(strRef, domainUrl))
                        {
                            Utility.Normalize(baseUri, ref strRef);
                            urls.Add(strRef);
                        }
                    }
                    catch (Exception)
                    {
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
            }

            //Logger.Info("Found: " + urls.Count + " ref(s)\r\n");

            return urls.ToArray();
        }


        static bool IsGoodUri(string strUri)
        {
            if (strUri.ToLower().StartsWith("javascript:"))
                return false;
            return true;
        }

        static bool IsDomainSpecified(string strUri, string domainUrl)
        {
            if (string.IsNullOrEmpty(domainUrl))
            {
                return true;
            }
            if (strUri.ToLower().Contains(domainUrl))
                return true;
            else
            {
                return false;
            }
        }

    }

    public class ContentFetcher
    {

        static ContentFetcher()
        {
            ThreadPool.SetMaxThreads(8, 8);
        }
        private static string domain = "";
        private static HashSet<string> globalList = new HashSet<string>();
        public static void SetUrlList(DataTable dt,string site)
        {
            domain = site;
            foreach (DataRow s in dt.Rows)
            {
                globalList.Add(s[0].ToString().Replace("$",site));
            }
        }

        public static void QueuePages(string url)
        {
            ThreadPool.QueueUserWorkItem(GetWebContent, url);
        }

        public static void GetWebContent(object url)
        {
            string address = (string)url;
            //NWebRequest req = new NWebRequest(new Uri(address), true);
            //req.Timeout = MemCache.ConnectionTimeoutMs;
            //NWebResponse response = req.GetResponse();
            HttpWebRequest request;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(address);
                request.Timeout = 60000; request.Headers.Set("Pragma", "no-cache");
            }
            catch (Exception e)
            {
                return;
            }
            HttpWebResponse response;
            while (true)
            {
                try
                {
                    response = ((HttpWebResponse)request.GetResponse());
                    break;
                }
                catch (TimeoutException e)
                {
                    ;
                }
            }


            Stream streamReceive = response.GetResponseStream();

            string contentType  = response.ContentType;

            if (!contentType.Contains("text/html") &&
                !MemCache.AllowAllMimeTypes &&
                !MemCache.AllowedFileTypes.Contains(contentType))
                return;
            
            long totalLength = response.ContentLength;
            byte[] buffer = new byte[totalLength];
            int len =0;
            try
            {
                while (len != totalLength)
                {
                    len += streamReceive.Read(buffer, len, (int)totalLength - len);
                    Thread.Sleep(1000);
                }
            }
            catch (Exception e)
            {
                return;
            }
            //byte[] buffer = new byte[len];
            //Array.Copy(buff,buffer,len);
            if (response.ContentEncoding== "gzip")
            {
                using (MemoryStream ms = new MemoryStream(buffer), tempMs = new MemoryStream())
                {

                    GZipStream deCompress = new GZipStream(ms, CompressionMode.Decompress);
                    deCompress.CopyTo(tempMs);
                    deCompress.Close();
                    buffer = tempMs.ToArray();
                }

            }
            response.Close();
            lock (lockSaveTODB)
            {
                if (IsUrlExistInTable("webpages", address))
                {
                    SyncContentToDB(address, contentType, buffer);
                }
                else
                {
                    SaveContentToDB(address, contentType, buffer);
                }
            }

            Encoding encoding = Encoding.UTF8;
            if (contentType.ToLower().Contains("gb2312"))
            {
                encoding = Encoding.GetEncoding("gb2312");
            }
            string html = encoding.GetString(buffer);
            string baseUri = Utility.GetBaseUri(address);
            string[] links = Parser.ExtractLinks(baseUri, html,domain);

            //if (Settings.DataStoreMode == "1")
            //{
            //    //SQLiteUtility.InsertToRepo(PageRank.calcPageRank(url),url, 0, "", buffer, DateTime.Now, DateTime.Now, 0, "", Environment.MachineName,links.Length);
            //    //BerkeleyDBUtil.GetBDB().AddKeyValue(url, html);
            //    //FileSystemUtility.StoreWebFile(url, buffer);
            //}
            //else
            //{
            //    //FileSystemUtility.StoreWebFile(url, buffer);
            //}


            foreach (string link in links)
            {
                if (globalList.Contains(link) || link.EndsWith(".js") ||
                    link.EndsWith(".css") ||
                    link.EndsWith(".jpeg") ||
                    link.EndsWith(".jpg") ||
                    link.EndsWith(".gif") ||
                    link.EndsWith(".png")
                    ) continue;
                // 加入队列
                if (IsUrlExistInTable("fasturltable",link))
                {
                    continue;
                }
                else
                {
                    InsertIntoFastUrlTable(link);
                    lock (lockObjForGlobalist)
                    {
                        globalList.Add(link);
                        ThreadPool.QueueUserWorkItem(GetWebContent, link);
                    }
                }
            }

        }

        private static object lockObjForGlobalist = new object();

        private static SqlConnection queryConn = new SqlConnection(@"Data Source=gene-f\bandik;database=CrawlPages;integrated security=SSPI");
        private static object lockUrlExist = new object();
        public static bool IsUrlExistInTable(string tableName, string url)
        {
            lock (lockUrlExist)
            {
                if (queryConn.State == ConnectionState.Closed)
                {
                    queryConn.Open();
                }

                SqlCommand getElement = new SqlCommand();
                getElement.CommandType = CommandType.Text;
                getElement.Connection = queryConn;
                string selectStringT = "select count(*) from [{0}] where url='{1}'";
                int result = 0;

                getElement.CommandText = string.Format(selectStringT, tableName, url);
                try
                {
                    result = (int)getElement.ExecuteScalar();

                }
                catch (Exception e)
                {
                    Console.WriteLine(e.Message);

                }
                if (result > 0)
                    return true;
                return false;
            }
        }

        public static bool InsertIntoFastUrlTable(string url)
        {
            SqlConnection sqlConn = new SqlConnection();
            sqlConn.ConnectionString = @"Data Source=gene-f\bandik;database=CrawlPages;integrated security=SSPI";
            sqlConn.Open();
            int result = 0;

            SqlCommand insertElement = new SqlCommand();
            insertElement.CommandType = CommandType.Text;
            insertElement.Connection = sqlConn;
            string selectStringT = "insert into [FastUrlTable](url) values('{0}')";

            insertElement.CommandText = string.Format(selectStringT, url);
            try
            {
                result =insertElement.ExecuteNonQuery();
                
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
            }
            sqlConn.Close();
            if (result == 1) return true;
            return false;
        }

        private static SqlConnection saveToDBConn = new SqlConnection(@"Data Source=gene-f\bandik;database=CrawlPages;integrated security=SSPI");
        private static object lockSaveTODB = new object();
        public static int SaveContentToDB(string url,string contentType,byte[] content)
        {
            if (saveToDBConn.State == ConnectionState.Executing)
            {
                Thread.Sleep(1000);
            }
            if (saveToDBConn.State == ConnectionState.Closed)
            {
                saveToDBConn.Open();
            }

            SqlCommand sqlcom = new SqlCommand();
            sqlcom.Connection = saveToDBConn;
            sqlcom.CommandType = CommandType.Text;
            string insertStringT = "insert into webpages(url,contenttype,cachedcontent,updatetime) values('{0}','{1}',@cachedContent,getdate())";
            sqlcom.Parameters.Add("@cachedContent", SqlDbType.VarBinary);
            string insertString = string.Empty;

            //SqlCommand getElement = new SqlCommand();
            //getElement.CommandType = CommandType.Text;
            //getElement.Connection = saveToDBConn;
            //string selectStringT = "select count(*) from webpages where url='{0}'";
            //int result = 0;


            //getElement.CommandText = string.Format(selectStringT, url);
            //try
            //{
            //    result = (int)getElement.ExecuteScalar();
            //}
            //catch (Exception e)
            //{
            //    Console.WriteLine(e.Message);
            //}
            //if (result > 0)
            //    return 1;

            sqlcom.Parameters["@cachedContent"].Value = content;
            insertString = string.Format(insertStringT, url, contentType);
            sqlcom.CommandText = insertString;
            try
            {
                sqlcom.ExecuteNonQuery();
                return 0;
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return -1;
            }
            
        }

        public static int SyncContentToDB(string url, string contentType, byte[] content)
        {
            if (saveToDBConn.State == ConnectionState.Executing)
            {
                Thread.Sleep(1000);
            }
            if (saveToDBConn.State == ConnectionState.Closed)
            {
                saveToDBConn.Open();
            }

            SqlCommand sqlcom = new SqlCommand();
            sqlcom.Connection = saveToDBConn;
            sqlcom.CommandType = CommandType.Text;
            string insertStringT = "update webpages set contenttype = '{0}' , cachedcontent = @cachedContent , updatetime = getdate() where url = '{1}'";
            sqlcom.Parameters.Add("@cachedContent", SqlDbType.VarBinary);
            string insertString = string.Empty;

            sqlcom.Parameters["@cachedContent"].Value = content;
            insertString = string.Format(insertStringT, contentType, url);
            sqlcom.CommandText = insertString;
            try
            {
                sqlcom.ExecuteNonQuery();
                return 0;
            }
            catch (Exception e)
            {
                Console.WriteLine(e.Message);
                return -1;
            }

        }
    }
}
